************************************************************
************************************************************
**														  **
**	Tanzania - Prices at Panel Level 					  **
**										   				  **
**										   				  **
************************************************************
************************************************************

clear
version 14
set more off
cap log close

gl OUT 	"${LOCDRIVE}\Results"
gl TEMP	"${LOCDRIVE}\Tempdata"

loc r1_hh hhid
loc r1_weight hh_weight
loc r1_indiv sbmemno
loc r2_hh y2_hhid
loc r2_weight y2_weight
loc r2_indiv indidy2
loc r3_hh y3_hhid
loc r3_weight y3_weight
loc r3_indiv indidy3

loc ea ea
loc strata strataid
loc admin0 macroregion
loc admin1 region 		
loc admin2 district  	
loc admin3 ward

log using "${TEMP}\PanelPrices_Tanzania", replace

loc gooddisagglist $gooddisagglist 
loc servicedisagglist $servicedisagglist

use "${TEMP}\Sample_r1fwd.dta", clear
tempfile sample_r1fwd
save `sample_r1fwd', replace

loc food_itemlist 101 102 103 104 105 106 107 108 109 110 111 112 201 202 203 204 205 206 207 401 501 502 601 602 603 701 702 703 801 802 803 804 807 808 809 901 902 301 302 303 704 1001 1002 1003 1004 1101 1102 1104 1106 1107 
loc service_itemlist  util comms recr transport food housing educ health misc 
loc good_itemlist cons util educ health


***************
** SL Prices **
***************

** prepare list of problem individual prices to create additional price instrument for aggregation bias check 

loc foodproblist ""
loc lfoodproblist ""
loc goodproblist ""
loc lgoodproblist ""
loc serviceproblist ""
loc lserviceproblist ""
foreach cat in food good service  {
	foreach item in ``cat'_itemlist' {
		loc `cat'problist ``cat'problist' pr_`cat'_`item'
		loc l`cat'problist `l`cat'problist' lpr_`cat'_`item' 
		}
	}


** First pull in the raw prices

import excel using "${LOCDRIVE}\Parameters\SL_prices", clear firstrow sheet("Prices_for_SL") 

rename Year year_int
rename Month month_int

* rename prices to match service categories and good categories below
g pr_good_alc = AlcoholTobaccoNarcotics
g pr_good_cons = CothingFootwear
g pr_good_util = HousingUtilitiesFuel
g pr_good_educ = Education
g pr_good_health = Health
g pr_service_util = HousingUtilitiesFuel
g pr_service_comms = Communication
g pr_service_recr = RecreationCulture
g pr_service_transport = Transport
g pr_service_food = RestaurantsHotels
g pr_service_housing = HousingUtilitiesFuel
g pr_service_educ = Education
g pr_service_health = Health
g pr_service_misc = Misc

tempfile prices_bymonth
save `prices_bymonth', replace


** import budget share data and merge with prices

use "${TEMP}\Tanzania_Panel_r1fwd.dta", clear

merge m:1 month_int year_int using `prices_bymonth', nogen keep(1 3) 

* Make a list of goods and services

foreach gs in good service {
	loc `gs'explist ""
	loc `gs'prlist ""
	foreach good in ``gs'disagglist' {
		loc `gs'explist ``gs'explist' exp_`good'
		loc `gs'prlist ``gs'prlist' pr_`good'
		}
	}
	
keep `r1_hh' round `admin3' `admin2' `admin1' `admin0' `goodexplist' `serviceexplist' `goodprlist' `serviceprlist'

tempfile sldataset_goodservice_wide
save `sldataset_goodservice_wide', replace /*for gcat test later*/

reshape long pr_ exp_ , i(`r1_hh' round `admin3' `admin2' `admin1' `admin0') j(gs_disagg) string

sort `r1_hh' round `admin3' `admin2' `admin1' `admin0' gs_disagg

tempfile sldataset_goodservice
save `sldataset_goodservice', replace 


** Form price index for goods

use `sldataset_goodservice', clear
keep if strpos(gs_disagg,"good_")

* Make within-group budget shares
bysort `r1_hh' round: egen exp_good_tot = total(exp_)
g w_ = exp_ / exp_good_tot

tempfile sldataset_good
save `sldataset_good', replace

* Base K factor
keep if round==1 & `admin0'==${admin0_base}
collapse (mean) w_, by(gs_disagg)
rename w_ w_base

tempfile w_good_base
save `w_good_base', replace

loc ngoods: word count `gooddisagglist' 

loc kgood = 1
forvalues j = 1/`ngoods' {
	display "j is `j'; kgood is `kgood'"
	loc gj = gs_disagg in `j'
	display "gs_disagg is `gj'"
	loc wgj = w_base in `j'
	loc kgood = `kgood' * (`wgj'^(-`wgj'))
	display "kgood is now `kgood'"
	}

	
* Now make the index 
use `sldataset_good', clear
merge m:1 gs_disagg using `w_good_base' /*bring in the base budget shares*/

g expression = w_ * (log(pr_) - log(w_))
g expression_alt = w_base * (log(pr_) - log(w_base))
replace expression = 0 if w_ ==0 /*if a household buys 2 of the 4 items in a group, set budget shares of the 2 not purchased to zero*/

collapse (sum) expression expression_alt exp_, by(`r1_hh' round `admin3' `admin2' `admin1' `admin0')
rename exp_ exp_good

g pr_good = exp(expression) / `kgood'
replace pr_good = exp(expression_alt) / `kgood' if exp_good == 0 /*if a household does not buy anything in a group, use base budget shares for the household.*/

g lpr_good = log(pr_good)

kdensity lpr_good
kdensity pr_good

keep `r1_hh' round exp_good lpr_good pr_good

*Retain prices for aggregation robustness check*
merge 1:1 `r1_hh' round using `sldataset_goodservice_wide', nogen keepusing(`goodproblist') 
foreach var in `goodproblist' {
	g l`var' = log(`var')
	drop `var'
	}

tempfile slprices_good
save `slprices_good', replace


** Form price index for services 

use `sldataset_goodservice', clear
keep if strpos(gs_disagg,"service_")

* Make within-group budget shares
bysort `r1_hh' round: egen exp_service_tot = total(exp_)
g w_ = exp_ / exp_service_tot

tempfile sldataset_service
save `sldataset_service', replace

* Base K factor
keep if round==1 & `admin0'==${admin0_base}
collapse (mean) w_, by(gs_disagg)
rename w_ w_base
tempfile w_service_base
save `w_service_base', replace

loc nservices: word count `servicedisagglist' 

loc kservice = 1
forvalues j = 1/`nservices' {
	display "j is `j'; kservice is `kservice'"
	loc gj = gs_disagg in `j'
	display "gs_disagg is `gj'"
	loc wsj = w_base in `j'
	loc kservice = `kservice' * (`wsj'^(-`wsj'))
	display "kservice is now `kservice'"
	}

	
* Now make the index 
use `sldataset_service', clear
merge m:1 gs_disagg using `w_service_base' /*bring in the base budget shares*/

g expression = w_ * (log(pr_) - log(w_))
g expression_alt = w_base * (log(pr_) - log(w_base))
replace expression = 0 if w_ ==0 /*if a household buys 2 of the 4 items in a group, set budget shares of the 2 not purchased to zero*/

collapse (sum) expression expression_alt exp_, by(`r1_hh' round `admin3' `admin2' `admin1' `admin0')
rename exp_ exp_service

g pr_service = exp(expression) / `kservice'
replace pr_service = exp(expression_alt) / `kservice' if exp_service == 0 /*if a household does not buy anything in a group, use base budget shares for the household.*/

g lpr_service = log(pr_service)


kdensity lpr_service
kdensity pr_service

keep `r1_hh' round exp_service lpr_service pr_service

*Retain prices for aggregation robustness check*
merge 1:1 `r1_hh' round using `sldataset_goodservice_wide', nogen keepusing(`serviceproblist') 
foreach var in `serviceproblist' {
	g l`var' = log(`var')
	drop `var'
	}
	
tempfile slprices_service
save `slprices_service', replace



*********************************************
** PRICE INDEX FORMATION - FOODS SUBGROUPS **
*********************************************

** Group the foods by groups (Wave 1)

use "${TEMP}/Foodcons_item_wk_W1.dta", clear

tab foodgroup
count if foodgroup==.

sort `r1_hh' itemcode
tempfile foods_fisher_raw_W1
save `foods_fisher_raw_W1', replace

	* Will use this to generate obs for each itemcode and food group for hhs that do not have consumption data 
	sort `r1_hh' itemcode
	g count = 1
	bysort `r1_hh' : g itemorder = sum(count)
	drop count 
	su itemorder
	loc nitems = r(max) /*59 items*/	
	collapse (median) itemcode, by(itemorder)
	
	g foodgroup = .
	replace foodgroup = 1 if inlist(itemcode,101,102,103,104,105,106,107,108,109,110,111,112,201,202,203,204,205,206,207)
	replace foodgroup = 2 if inlist(itemcode,401,501,503,504,502,601,602,603,701,702,703)
	replace foodgroup = 3 if inlist(itemcode,801,802,803,804,805,806,807,808,809,810,901,902,903)
	replace foodgroup = 4 if inlist(itemcode,301,302,303,704,1001,1002,1003,1004,1101,1102,1103,1104,1105,1106,1107,1108)

	tab itemcode itemorder
	tempfile itemcodematch
	save `itemcodematch', replace


** Group the foods by groups (Wave 2)

use "${TEMP}/Foodcons_item_wk_W2.dta", clear

tab foodgroup
count if foodgroup==.

sort `r2_hh' itemcode
tempfile foods_fisher_raw_W2
save `foods_fisher_raw_W2', replace


** Group the foods by groups (Wave 3)

use "${TEMP}/Foodcons_item_wk_W3.dta", clear

tab foodgroup
count if foodgroup==.

sort `r3_hh' itemcode
tempfile foods_fisher_raw_W3
save `foods_fisher_raw_W3', replace


** Append the rounds 

forvalues r = 1/3 {
    use `sample_r1fwd', clear
	keep if round==`r'
	sort `r`r'_hh'
	
	expand `nitems'
	g count = 1
	bysort `r`r'_hh' : g itemorder= sum(count) 
	merge m:1 itemorder using `itemcodematch', keepusing(itemcode foodgroup)
	drop itemorder

	merge 1:1 `r`r'_hh' itemcode using `foods_fisher_raw_W`r'', nogen keep(1 3) keepusing(q_total itemcode foodgroup price_med)
		* r1 master sample has 37 obs not matched in the food data  
		* r2 master sample has 60 obs not matched in the food data
		* r3 master sample has 106 obs not matched in the food data 
	keep `r1_hh' round itemcode foodgroup q_total price_med `admin0' `admin1' `admin2' `admin3'
	
	tempfile foods_fisher_raw_W`r'
	save `foods_fisher_raw_W`r'', replace
	}
	

clear
forvalues r=1/3 {
    append using `foods_fisher_raw_W`r''
	}
	

** Replace missing values

merge m:1 `r1_hh' round using "${TEMP}\Tanzania_Panel_r1fwd.dta", nogen keep(1 3) keepusing(month_int)

bysort itemcode round month_int `admin0' `admin1' `admin2' `admin3': egen price_med_adm3 = median(price_med)
bysort itemcode round month_int `admin0' `admin1' `admin2' `admin3': egen n_price_med_adm3 = total(!missing(price_med))
replace price_med_adm3=. if n_price_med_adm3 < 3

bysort itemcode round month_int `admin0' `admin1' `admin2': egen price_med_adm2 = median(price_med)
bysort itemcode round month_int `admin0' `admin1' `admin2': egen n_price_med_adm2 = total(!missing(price_med))
replace price_med_adm2=. if n_price_med_adm2 < 3

bysort itemcode round month_int `admin0' `admin1': egen price_med_adm1 = median(price_med)
bysort itemcode round month_int `admin0' `admin1': egen n_price_med_adm1 = total(!missing(price_med))
replace price_med_adm1=. if n_price_med_adm1 < 3

bysort itemcode round month_int `admin0': egen price_med_adm0 = median(price_med)
bysort itemcode round month_int `admin0': egen n_price_med_adm0 = total(!missing(price_med))
replace price_med_adm0=. if n_price_med_adm0 < 3

replace price_med = price_med_adm3 if price_med==. & price_med_adm3 != .
replace price_med = price_med_adm2 if price_med==. & price_med_adm2 != .
replace price_med = price_med_adm1 if price_med==. & price_med_adm1 != .
replace price_med = price_med_adm0 if price_med==. & price_med_adm0 != .

bysort round item: count if price_med==.

drop price_med_adm3 price_med_adm2 price_med_adm1 price_med_adm0 n_price_med_* month_int

tempfile foods_fisher_raw
save `foods_fisher_raw', replace 


* Create a wide version of the food data
use `foods_fisher_raw', clear 
rename price_med pr_food_
drop foodgroup q_total

reshape wide pr_food_, i(`r1_hh' round) j(itemcode)
tempfile foods_fisher_raw_wide
save `foods_fisher_raw_wide', replace


** Get the base prices 

use `foods_fisher_raw', clear

keep if round==1 & `admin0'==${admin0_base}
collapse (mean) qty0_adm0=q_total pr_kg_pred0_adm0=price_med, by(itemcode foodgroup)

tempfile base_qty_price_admin0
save `base_qty_price_admin0', replace


** Construct fisher price at hh level - food in aggregate
use `foods_fisher_raw', clear

merge m:1 itemcode foodgroup using `base_qty_price_admin0', nogen keep(1 3) /*all are _m=3*/ 

g laspeyeres_num = price_med * qty0_adm0
g laspeyeres_denom = pr_kg_pred0_adm0 * qty0_adm0
g paasche_num = price_med * q_total
g paasche_denom = pr_kg_pred0_adm0 * q_total 

collapse (sum) laspeyeres_num laspeyeres_denom paasche_num paasche_denom, by(`r1_hh' `admin3' `admin2' `admin1' `admin0' round)
isid  round `r1_hh' `admin3' `admin2' `admin1' `admin0'

g pr_food = sqrt((laspeyeres_num/laspeyeres_denom)*(paasche_num/paasche_denom))
g lpr_food = log(pr_food)

tempfile prices_food_hh
save `prices_food_hh', replace

collapse (median) lpr_food_adm3=lpr_food, by(round `admin3' `admin2' `admin1' `admin0')
isid round `admin3' `admin2' `admin1' `admin0'
tempfile prices_food_adm3
save `prices_food_adm3', replace

use `prices_food_hh', clear
collapse (median) lpr_food_adm2=lpr_food, by(round `admin2' `admin1' `admin0')
isid round `admin2' `admin1' `admin0'
tempfile prices_food_adm2
save `prices_food_adm2', replace

use `prices_food_hh', clear
collapse (median) lpr_food_adm1=lpr_food, by(round `admin1' `admin0')
isid round `admin1' `admin0'
tempfile prices_food_adm1
save `prices_food_adm1', replace

* make the panel sample like the food price sample
use `sample_r1fwd', clear
isid `r1_hh' round  

merge 1:1 round `r1_hh' `admin3' `admin2' `admin1' `admin0' using `prices_food_hh', nogen keep(1 3)
merge m:1 round `admin3' `admin2' `admin1' `admin0' using `prices_food_adm3', nogen keep(1 3)
merge m:1 round `admin2' `admin1' `admin0' using `prices_food_adm2', nogen keep(1 3)
merge m:1 round `admin1' `admin0' using `prices_food_adm1', nogen keep(1 3)

count if lpr_food==. /**/
replace lpr_food = lpr_food_adm3 if lpr_food==. & lpr_food_adm3 != 0 /**/
replace lpr_food = lpr_food_adm2 if lpr_food==. & lpr_food_adm2 != 0 /**/
replace lpr_food = lpr_food_adm1 if lpr_food==. & lpr_food_adm1 != 0 /**/

keep `r1_hh' round lpr_food
sort `r1_hh' round 
isid `r1_hh' round

*Retain prices for aggregation robustness check*

merge 1:1 `r1_hh' round using `foods_fisher_raw_wide', nogen keepusing(`foodproblist')  

foreach var in `foodproblist' {
	g l`var' = log(`var')	
	}
	
tempfile prices_food
save `prices_food', replace


***********************
** PRICE INSTRUMENTS **
***********************

** good, service, food 
forvalues r = 1/3 {
	use `sample_r1fwd', clear
	keep if round==`r'
*	merge 1:1 `r1_hh' round using `prices_foodgroups', nogen keep(1 3) keepusing(lpr_food1 lpr_food2 lpr_food3 lpr_food4)
	merge 1:1 `r1_hh' round using `prices_food', nogen keep(1 3) keepusing(lpr_food) 
	merge 1:1 `r1_hh' round using `slprices_good', nogen keep(1 3) keepusing(lpr_good) 
	merge 1:1 `r1_hh' round using `slprices_service', nogen keep(1 3) keepusing(lpr_service) 
	merge 1:1 `r1_hh' round using `"${TEMP}\Tanzania_Panel_r1fwd.dta"', nogen keep(1 3) keepusing(month_int year_int)
	
	cap drop admin0_c admin1_c admin2_c admin3_c `admin0' `admin1' `admin2' `admin3' `ea'
	merge 1:1 `r`r'_hh' using "${TEMP}/Sample_links_R`r'.dta", nogen keep(1 3) keepusing(admin0_c admin1_c admin2_c admin3_c)

	foreach good in /*food1 food2 food3 food4*/ food good service {
		g lpr_`good'_i1 = .
		g lpr_`good'_i2 = .
		g lpr_`good'_i3 = .
		}
	
	count 
	loc N = r(N)
	
	quietly {
		forvalues i=1/`N' {
			display "`i'"
			
			cap drop obsadm1 obsadm2 obsadm3 obshh timing_weight timing_diff timing_weight_moy
			
			g obsadm1 = admin1_c[`i']
			g obsadm2 = admin2_c[`i']
			g obsadm3 = admin3_c[`i']
			g obshh = `r`r'_hh'[`i']
		
			g timing_diff = abs((month_int - month_int[`i']))*(365/12) + abs((year_int - year_int[`i']))*356
			g timing_weight = 1/timing_diff
			g timing_weight_moy = 1/(abs((month_int - month_int[`i']))*(365/12))

			foreach good in /*food1 food2 food3 food4*/ food good service {
				display "`good'"
				* first instrument: same district but different ward, using weights for time difference
				su lpr_`good' if admin2_c==obsadm2 & admin3_c!=obsadm3 [aweight=timing_weight]
				loc count=r(N)
				if `count'>0 {
					replace lpr_`good'_i1 = r(mean) in `i'
					}
				
				* second instrument: same zone but different ward and surveyed within a 30-day period
				su lpr_`good' if admin1_c==obsadm1 & admin3_c!=obsadm3 & timing_diff <= 365/6 [aweight=timing_weight]
				loc count=r(N)
				if `count'>0 {
					replace lpr_`good'_i2 = r(mean) in `i'
					}
					
				* third instrument: donor households in the same ward excluding the hh -- I do not end up using this, I used the residualized price as the 3rd instrument instead
				su lpr_`good' if admin3_c==obsadm3 & `r`r'_hh'!=obshh [aweight=timing_weight_moy]
				loc count=r(N)
				if `count'>0 {
					replace lpr_`good'_i3 = r(mean) in `i'
					}
				}
			}
		}

	** replace missing values with the mean across instruments
	foreach good in /*food1 food2 food3 food4*/ food good service {
		egen lpr_`good'_imean = rowmean(lpr_`good'_i1 lpr_`good'_i2 lpr_`good'_i3)
		forvalues i = 1/2 /*3*/ {
			count if lpr_`good'_i`i'==.
			replace lpr_`good'_i`i'=lpr_`good'_imean if lpr_`good'_i`i'==. & lpr_`good'_imean!=.
			count if lpr_`good'_i`i'==.
			}
		}
	
	keep `r1_hh' round /*lpr_food1 lpr_food2 lpr_food3 lpr_food4*/ lpr_food lpr_good lpr_service lpr_*i1 lpr_*i2 lpr_*i3
	
	tempfile price_instruments_r`r'
	save `price_instruments_r`r'', replace
	}

clear
forvalues r=1/3 {
    append using `price_instruments_r`r''
	}

tempfile prices_with_instruments
save `prices_with_instruments', replace


** Now look at aggregation bias instruments 

use `sample_r1fwd', clear
merge 1:1 `r1_hh' round using `prices_food', nogen keep(1 3) keepusing(lpr_food `lfoodproblist') 
merge 1:1 `r1_hh' round using `slprices_good', nogen keep(1 3) keepusing(lpr_good `lgoodproblist') 
merge 1:1 `r1_hh' round using `slprices_service', nogen keep(1 3) keepusing(lpr_service `lserviceproblist')
merge 1:1 `r1_hh' round using `"${TEMP}\Tanzania_Panel_r1fwd.dta"', nogen keep(1 3) keepusing(month_int year_int)


foreach cat in food good service {
	loc l`cat'problist_rat ""
	foreach var in `l`cat'problist' {
		display "Variable `var':"
		count if `var'==.
		display "`var' missing: `r(N)'"
		bysort round month_int: egen `var'_med_month = median(`var')
		replace `var' = `var'_med_month if `var'==. & `var'_med_month != .
		bysort round: egen `var'_med_round = median(`var')
		replace `var' = `var'_med_round if `var'==. & `var'_med_round != .
		count if `var'==.
		display "`var' missing: `r(N)'"

		drop `var'_med_month `var'_med_round
		
		g `var'_rat = `var' - lpr_`cat'
		loc l`cat'problist_rat `l`cat'problist_rat' `var'_rat
		}
	}
	
	regress lpr_food i.round `lfoodproblist_rat'
	predict pred 
	g lpr_food_i4 = lpr_food - pred
	drop pred
	
	* now residualize goods
	regress lpr_good i.round `lgoodproblist_rat'
	predict pred
	g lpr_good_i4 = lpr_good - pred
	drop pred
	
	* now residualize services
	regress lpr_service i.round `lserviceproblist_rat'
	predict pred
	g lpr_service_i4 = lpr_service - pred
	drop pred

keep `r1_hh' round lpr_*i4

tempfile aggregation_instruments
save `aggregation_instruments', replace


******************
** COMPILE DATA **
******************

use "${TEMP}\Tanzania_Panel_r1fwd.dta"
sort `r1_hh' round
merge 1:1 `r1_hh' round using `prices_with_instruments', nogen keep(1 3)
sort `r1_hh' round
merge 1:1 `r1_hh' round using `aggregation_instruments', nogen keep(1 3) 
sort `r1_hh' round 


* price variables: 
la var lpr_food "Food prices"	
la var lpr_good "Goods prices"
la var lpr_service "Service prices"

save "${TEMP}\Tanzania_Panel_estready_all.dta", replace


foreach i in 1 2 3 4 {
	display "Good: i`i':"
	regress lpr_good lpr_good_i`i' i.round
	display "Service: i`i':"
	regress lpr_service lpr_service_i`i' i.round
	display "Food: i`i' :"
	regress lpr_food lpr_food_i`i' i.round
	}


***************************************************
** Generalized Commidty Aggregation Theorem Test **
***************************************************

use "${TEMP}\Tanzania_Panel_estready_all.dta", clear

cap drop _m
sort `r1_hh' round
merge 1:1 `r1_hh' round using `slprices_good', nogen keepusing(pr_good) /*this merge is good*/

sort `r1_hh' round
merge 1:1 `r1_hh' round using `slprices_service', nogen keepusing(pr_service) /*this merge is good*/

sort `r1_hh' round
merge 1:1 `r1_hh' round using `sldataset_goodservice_wide', nogen keepusing(`goodprlist' `serviceprlist') /*this merge is good*/

sort `r1_hh' round
merge 1:1 `r1_hh' round using `foods_fisher_raw_wide', keepusing(pr_food*) /*this merge is good*/

sort `r1_hh' round 
merge 1:1 `r1_hh' round using `prices_food', nogen keep(1 3) keepusing(lpr_food)

foreach var in food {
	g pr_`var' = exp(lpr_`var')
	}
	
*Create a month variable in Stata format
g month_ts = (year_int - 1960)*12 + month_int

*Create a unique panel identifier
egen panelid = group(`r1_hh' round)


* Generate ratios
/*Grayed out food items were dropped from aggregates because they are too rarely consuemd -- moved this item list to the top of the do file */

*loc service_itemlist  util comms recr transport food housing educ health misc 
*loc good_itemlist cons util educ health
*loc food_itemlist 101 102 103 104 105 106 107 108 109 110 111 112 201 202 203 204 205 206 207 401 501 502 /*503*/ /*504*/ 601 602 603 701 702 703 801 802 803 804 /*805*/ /*806*/ 807 808 809 /*810*/ 901 902 /*903*/ 301 302 303 704 1001 1002 1003 1004 1101 1102 /*1103*/ 1104 /*1105*/ 1106 1107 /*1108*/

foreach cat in food good service {
	loc `cat'_prratlist ""
	foreach item in ``cat'_itemlist' {
		cap drop lpr_`cat'_`item'_rat
		g lpr_`cat'_`item'_rat = log(pr_`cat'_`item' / pr_`cat')
		loc `cat'_prratlist ``cat'_prratlist' lpr_`cat'_`item'_rat
		}
		
	display "``cat'_prratlist'"
	}


egen admin1_id = group(`admin0' `admin1')
egen admin0_id = group(`admin0')

xtset hh_c round 

tempfile allprices
save `allprices', replace

save "${TEMP}\allprices_ggattest.dta", replace


* Collapse by month and admin1 

use "${TEMP}\allprices_ggattest.dta", clear

collapse (mean) lpr_service `service_prratlist' lpr_good `good_prratlist' lpr_food `food_prratlist', by(admin0_id month_ts)
drop if month_ts==.

xtset, clear
xtset admin0_id month_ts

tempfile allprices_monthpanel_admin0
save `allprices_monthpanel_admin0', replace

save "${TEMP}\allprices_ggattest_monthpanel_admin0.dta", replace


* Collapse by month only

use "${TEMP}\allprices_ggattest.dta", clear

collapse (mean) lpr_service `service_prratlist' lpr_good `good_prratlist' lpr_food `food_prratlist', by(month_ts)
drop if month_ts==.

xtset, clear
tsset month_ts

tempfile allprices_monthpanel
save `allprices_monthpanel', replace

save "${TEMP}\allprices_ggattest_monthpanel.dta", replace

	
** Test for unit root (month level - collapsed) 

use `allprices_monthpanel', clear

foreach cat in food good service  {
	dfuller lpr_`cat'
	loc dft_`cat' = r(p)

	display "``cat'_itemlist'"
	foreach item in ``cat'_itemlist' {
		cap dfuller lpr_`cat'_`item'_rat
		if _rc==0 {
			loc dft_`cat'_`item' = r(p)
			}
			else if _rc!=0 {
				loc dft_`cat'_`item' = "error" 
				}
		}
	}


foreach cat in food good service {
	if `dft_`cat'' <= 0.05 {
		loc test_`cat' "SR" /*reject the augmented dicky fuller test it is stationary so would use spearman rank*/
		}
		else if `dft_`cat'' > 0.05 {
			loc test_`cat' "CI" /*reject the augmented dicky fuller test it is unit root so would use cointegration test*/
			}
	display "`cat': Test `test_`cat''"
	
	loc `cat'_sr_count = 0
	loc `cat'_ci_count = 0

	foreach item in ``cat'_itemlist' {
		if "`dft_`cat'_`item''"=="error" {
			loc test_`cat'_`item' `test_`cat''
			}
			else if (`dft_`cat'_`item'' <= 0.05 & `dft_`cat'' > 0.05) | (`dft_`cat'_`item'' > 0.05 & `dft_`cat'' <= 0.05) {
				loc test_`cat'_`item' "NO" /*no test*/
				}
				else if `dft_`cat'_`item'' > 0.05 & `dft_`cat'' > 0.05 {
					loc test_`cat'_`item' "CI" /*cointegration test*/ 
					loc `cat'_ci_count = ``cat'_ci_count'+1
					}
					else if `dft_`cat'_`item'' <= 0.05 & `dft_`cat'' <= 0.05 {
						loc test_`cat'_`item' "SR"  /*spearmans rank correlation test*/
						loc `cat'_sr_count = ``cat'_sr_count'+1
						}
		display "`cat', `item':"
		display "pval: `dft_`cat'_`item''" 
		display "test: `test_`cat'_`item''"
		}
	display "`cat' CI count: ``cat'_ci_count'"
	display "`cat' SR count: ``cat'_sr_count'"
	}

foreach cat in food good service {
	foreach item in ``cat'_itemlist' {
		display "`cat', `item' test: `test_`cat'_`item''"
		if "`test_`cat'_`item''"=="CI" {
			cap mat drop gstats
			var lpr_`cat' lpr_`cat'_`item'_`rat'
			vargranger
			mat gstats = r(gstats)
			loc rawpval = gstats[1,3]
			loc `cat'_`item'_sr_bpval = min((`rawpval'*``cat'_ci_count'),1)
			display "Raw p-val: `rawpval'. Bonferonni corrected p-val: ``cat'_`item'_sr_bpval'"
			}
			else if "`test_`cat'_`item''"=="SR" {
				spearman lpr_`cat' lpr_`cat'_`item'_rat
				loc rawpval = r(p)
				loc `cat'_`item'_sr_bpval = min((`rawpval'*``cat'_sr_count'),1)
				loc `cat'_`item'_sr_rho = r(rho)
				display "Raw p-val: `rawpval'. Bonferonni corrected p-val: ``cat'_`item'_sr_bpval'"
				}
		}
	}



log close